{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "931f7017-d2e2-4b1a-8fdf-9ddf119fc8ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import necessary libraries of pydantic and the llama-cpp-agent framework.\n",
    "from enum import Enum\n",
    "from typing import List\n",
    "from langchain_community.chat_models import ChatLlamaCpp\n",
    "from langchain_community.embeddings import LlamaCppEmbeddings\n",
    "from langchain_community.vectorstores import SQLiteVec\n",
    "from langchain_core.messages import HumanMessage, SystemMessage\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40f13f0a-9f41-4ad4-ab9b-494509558a35",
   "metadata": {},
   "source": [
    "# 使用chartlamacpp加载DeepSeek量化推理型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "059444fc-6f34-4403-bea2-d19b6457b792",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no\n",
      "ggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\n",
      "ggml_cuda_init: found 1 CUDA devices:\n",
      "  Device 0: NVIDIA GeForce RTX 3060, compute capability 8.6, VMM: yes\n",
      "llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3060) - 11796 MiB free\n",
      "llama_model_loader: loaded meta data with 27 key-value pairs and 339 tensors from /home/spike/code/AI/deepseek-r1-distill-qwen-7b-q4_k_m.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = qwen2\n",
      "llama_model_loader: - kv   1:                               general.type str              = model\n",
      "llama_model_loader: - kv   2:                               general.name str              = DeepSeek R1 Distill Qwen 7B\n",
      "llama_model_loader: - kv   3:                           general.basename str              = DeepSeek-R1-Distill-Qwen\n",
      "llama_model_loader: - kv   4:                         general.size_label str              = 7B\n",
      "llama_model_loader: - kv   5:                            general.license str              = mit\n",
      "llama_model_loader: - kv   6:                          qwen2.block_count u32              = 28\n",
      "llama_model_loader: - kv   7:                       qwen2.context_length u32              = 131072\n",
      "llama_model_loader: - kv   8:                     qwen2.embedding_length u32              = 3584\n",
      "llama_model_loader: - kv   9:                  qwen2.feed_forward_length u32              = 18944\n",
      "llama_model_loader: - kv  10:                 qwen2.attention.head_count u32              = 28\n",
      "llama_model_loader: - kv  11:              qwen2.attention.head_count_kv u32              = 4\n",
      "llama_model_loader: - kv  12:                       qwen2.rope.freq_base f32              = 10000.000000\n",
      "llama_model_loader: - kv  13:     qwen2.attention.layer_norm_rms_epsilon f32              = 0.000001\n",
      "llama_model_loader: - kv  14:                       tokenizer.ggml.model str              = gpt2\n",
      "llama_model_loader: - kv  15:                         tokenizer.ggml.pre str              = deepseek-r1-qwen\n",
      "llama_model_loader: - kv  16:                      tokenizer.ggml.tokens arr[str,152064]  = [\"!\", \"\\\"\", \"#\", \"$\", \"%\", \"&\", \"'\", ...\n",
      "llama_model_loader: - kv  17:                  tokenizer.ggml.token_type arr[i32,152064]  = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  18:                      tokenizer.ggml.merges arr[str,151387]  = [\"Ġ Ġ\", \"ĠĠ ĠĠ\", \"i n\", \"Ġ t\",...\n",
      "llama_model_loader: - kv  19:                tokenizer.ggml.bos_token_id u32              = 151646\n",
      "llama_model_loader: - kv  20:                tokenizer.ggml.eos_token_id u32              = 151643\n",
      "llama_model_loader: - kv  21:            tokenizer.ggml.padding_token_id u32              = 151643\n",
      "llama_model_loader: - kv  22:               tokenizer.ggml.add_bos_token bool             = true\n",
      "llama_model_loader: - kv  23:               tokenizer.ggml.add_eos_token bool             = false\n",
      "llama_model_loader: - kv  24:                    tokenizer.chat_template str              = {% if not add_generation_prompt is de...\n",
      "llama_model_loader: - kv  25:               general.quantization_version u32              = 2\n",
      "llama_model_loader: - kv  26:                          general.file_type u32              = 15\n",
      "llama_model_loader: - type  f32:  141 tensors\n",
      "llama_model_loader: - type q4_K:  169 tensors\n",
      "llama_model_loader: - type q6_K:   29 tensors\n",
      "print_info: file format = GGUF V3 (latest)\n",
      "print_info: file type   = Q4_K - Medium\n",
      "print_info: file size   = 4.36 GiB (4.91 BPW) \n",
      "init_tokenizer: initializing tokenizer for type 2\n",
      "load: control token: 151660 '<|fim_middle|>' is not marked as EOG\n",
      "load: control token: 151659 '<|fim_prefix|>' is not marked as EOG\n",
      "load: control token: 151653 '<|vision_end|>' is not marked as EOG\n",
      "load: control token: 151645 '<｜Assistant｜>' is not marked as EOG\n",
      "load: control token: 151644 '<｜User｜>' is not marked as EOG\n",
      "load: control token: 151655 '<|image_pad|>' is not marked as EOG\n",
      "load: control token: 151651 '<|quad_end|>' is not marked as EOG\n",
      "load: control token: 151646 '<｜begin▁of▁sentence｜>' is not marked as EOG\n",
      "load: control token: 151643 '<｜end▁of▁sentence｜>' is not marked as EOG\n",
      "load: control token: 151652 '<|vision_start|>' is not marked as EOG\n",
      "load: control token: 151647 '<|EOT|>' is not marked as EOG\n",
      "load: control token: 151654 '<|vision_pad|>' is not marked as EOG\n",
      "load: control token: 151656 '<|video_pad|>' is not marked as EOG\n",
      "load: control token: 151661 '<|fim_suffix|>' is not marked as EOG\n",
      "load: control token: 151650 '<|quad_start|>' is not marked as EOG\n",
      "load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n",
      "load: special tokens cache size = 22\n",
      "load: token to piece cache size = 0.9310 MB\n",
      "print_info: arch             = qwen2\n",
      "print_info: vocab_only       = 0\n",
      "print_info: n_ctx_train      = 131072\n",
      "print_info: n_embd           = 3584\n",
      "print_info: n_layer          = 28\n",
      "print_info: n_head           = 28\n",
      "print_info: n_head_kv        = 4\n",
      "print_info: n_rot            = 128\n",
      "print_info: n_swa            = 0\n",
      "print_info: n_embd_head_k    = 128\n",
      "print_info: n_embd_head_v    = 128\n",
      "print_info: n_gqa            = 7\n",
      "print_info: n_embd_k_gqa     = 512\n",
      "print_info: n_embd_v_gqa     = 512\n",
      "print_info: f_norm_eps       = 0.0e+00\n",
      "print_info: f_norm_rms_eps   = 1.0e-06\n",
      "print_info: f_clamp_kqv      = 0.0e+00\n",
      "print_info: f_max_alibi_bias = 0.0e+00\n",
      "print_info: f_logit_scale    = 0.0e+00\n",
      "print_info: f_attn_scale     = 0.0e+00\n",
      "print_info: n_ff             = 18944\n",
      "print_info: n_expert         = 0\n",
      "print_info: n_expert_used    = 0\n",
      "print_info: causal attn      = 1\n",
      "print_info: pooling type     = 0\n",
      "print_info: rope type        = 2\n",
      "print_info: rope scaling     = linear\n",
      "print_info: freq_base_train  = 10000.0\n",
      "print_info: freq_scale_train = 1\n",
      "print_info: n_ctx_orig_yarn  = 131072\n",
      "print_info: rope_finetuned   = unknown\n",
      "print_info: ssm_d_conv       = 0\n",
      "print_info: ssm_d_inner      = 0\n",
      "print_info: ssm_d_state      = 0\n",
      "print_info: ssm_dt_rank      = 0\n",
      "print_info: ssm_dt_b_c_rms   = 0\n",
      "print_info: model type       = 7B\n",
      "print_info: model params     = 7.62 B\n",
      "print_info: general.name     = DeepSeek R1 Distill Qwen 7B\n",
      "print_info: vocab type       = BPE\n",
      "print_info: n_vocab          = 152064\n",
      "print_info: n_merges         = 151387\n",
      "print_info: BOS token        = 151646 '<｜begin▁of▁sentence｜>'\n",
      "print_info: EOS token        = 151643 '<｜end▁of▁sentence｜>'\n",
      "print_info: EOT token        = 151643 '<｜end▁of▁sentence｜>'\n",
      "print_info: PAD token        = 151643 '<｜end▁of▁sentence｜>'\n",
      "print_info: LF token         = 198 'Ċ'\n",
      "print_info: FIM PRE token    = 151659 '<|fim_prefix|>'\n",
      "print_info: FIM SUF token    = 151661 '<|fim_suffix|>'\n",
      "print_info: FIM MID token    = 151660 '<|fim_middle|>'\n",
      "print_info: FIM PAD token    = 151662 '<|fim_pad|>'\n",
      "print_info: FIM REP token    = 151663 '<|repo_name|>'\n",
      "print_info: FIM SEP token    = 151664 '<|file_sep|>'\n",
      "print_info: EOG token        = 151643 '<｜end▁of▁sentence｜>'\n",
      "print_info: EOG token        = 151662 '<|fim_pad|>'\n",
      "print_info: EOG token        = 151663 '<|repo_name|>'\n",
      "print_info: EOG token        = 151664 '<|file_sep|>'\n",
      "print_info: max token length = 256\n",
      "load_tensors: loading model tensors, this can take a while... (mmap = true)\n",
      "load_tensors: layer   0 assigned to device CUDA0\n",
      "load_tensors: layer   1 assigned to device CUDA0\n",
      "load_tensors: layer   2 assigned to device CUDA0\n",
      "load_tensors: layer   3 assigned to device CUDA0\n",
      "load_tensors: layer   4 assigned to device CUDA0\n",
      "load_tensors: layer   5 assigned to device CUDA0\n",
      "load_tensors: layer   6 assigned to device CUDA0\n",
      "load_tensors: layer   7 assigned to device CUDA0\n",
      "load_tensors: layer   8 assigned to device CUDA0\n",
      "load_tensors: layer   9 assigned to device CUDA0\n",
      "load_tensors: layer  10 assigned to device CUDA0\n",
      "load_tensors: layer  11 assigned to device CUDA0\n",
      "load_tensors: layer  12 assigned to device CUDA0\n",
      "load_tensors: layer  13 assigned to device CUDA0\n",
      "load_tensors: layer  14 assigned to device CUDA0\n",
      "load_tensors: layer  15 assigned to device CUDA0\n",
      "load_tensors: layer  16 assigned to device CUDA0\n",
      "load_tensors: layer  17 assigned to device CUDA0\n",
      "load_tensors: layer  18 assigned to device CUDA0\n",
      "load_tensors: layer  19 assigned to device CUDA0\n",
      "load_tensors: layer  20 assigned to device CUDA0\n",
      "load_tensors: layer  21 assigned to device CUDA0\n",
      "load_tensors: layer  22 assigned to device CUDA0\n",
      "load_tensors: layer  23 assigned to device CUDA0\n",
      "load_tensors: layer  24 assigned to device CUDA0\n",
      "load_tensors: layer  25 assigned to device CUDA0\n",
      "load_tensors: layer  26 assigned to device CUDA0\n",
      "load_tensors: layer  27 assigned to device CUDA0\n",
      "load_tensors: layer  28 assigned to device CUDA0\n",
      "load_tensors: tensor 'token_embd.weight' (q4_K) (and 0 others) cannot be used with preferred buffer type CPU_AARCH64, using CPU instead\n",
      "load_tensors: offloading 28 repeating layers to GPU\n",
      "load_tensors: offloading output layer to GPU\n",
      "load_tensors: offloaded 29/29 layers to GPU\n",
      "load_tensors:        CUDA0 model buffer size =  4168.09 MiB\n",
      "load_tensors:   CPU_Mapped model buffer size =   292.36 MiB\n",
      "..................................................................................\n",
      "llama_init_from_model: n_batch is less than GGML_KQ_MASK_PAD - increasing to 64\n",
      "llama_init_from_model: n_seq_max     = 1\n",
      "llama_init_from_model: n_ctx         = 4096\n",
      "llama_init_from_model: n_ctx_per_seq = 4096\n",
      "llama_init_from_model: n_batch       = 64\n",
      "llama_init_from_model: n_ubatch      = 8\n",
      "llama_init_from_model: flash_attn    = 0\n",
      "llama_init_from_model: freq_base     = 10000.0\n",
      "llama_init_from_model: freq_scale    = 1\n",
      "llama_init_from_model: n_ctx_per_seq (4096) < n_ctx_train (131072) -- the full capacity of the model will not be utilized\n",
      "llama_kv_cache_init: kv_size = 4096, offload = 1, type_k = 'f16', type_v = 'f16', n_layer = 28, can_shift = 1\n",
      "llama_kv_cache_init: layer 0: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 1: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 2: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 3: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 4: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 5: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 6: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 7: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 8: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 9: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 10: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 11: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 12: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 13: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 14: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 15: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 16: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 17: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 18: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 19: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 20: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 21: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 22: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 23: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 24: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 25: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 26: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init: layer 27: n_embd_k_gqa = 512, n_embd_v_gqa = 512\n",
      "llama_kv_cache_init:      CUDA0 KV buffer size =   224.00 MiB\n",
      "llama_init_from_model: KV self size  =  224.00 MiB, K (f16):  112.00 MiB, V (f16):  112.00 MiB\n",
      "llama_init_from_model:  CUDA_Host  output buffer size =     0.58 MiB\n",
      "llama_init_from_model:      CUDA0 compute buffer size =     4.94 MiB\n",
      "llama_init_from_model:  CUDA_Host compute buffer size =     1.11 MiB\n",
      "llama_init_from_model: graph nodes  = 986\n",
      "llama_init_from_model: graph splits = 2\n",
      "CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | \n",
      "Model metadata: {'general.file_type': '15', 'tokenizer.ggml.add_eos_token': 'false', 'tokenizer.ggml.add_bos_token': 'true', 'tokenizer.ggml.bos_token_id': '151646', 'qwen2.attention.layer_norm_rms_epsilon': '0.000001', 'general.architecture': 'qwen2', 'tokenizer.ggml.padding_token_id': '151643', 'general.basename': 'DeepSeek-R1-Distill-Qwen', 'qwen2.embedding_length': '3584', 'tokenizer.ggml.pre': 'deepseek-r1-qwen', 'general.name': 'DeepSeek R1 Distill Qwen 7B', 'qwen2.block_count': '28', 'general.type': 'model', 'general.size_label': '7B', 'general.license': 'mit', 'qwen2.context_length': '131072', 'tokenizer.chat_template': \"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\\\n' + '```json' + '\\\\n' + tool['function']['arguments'] + '\\\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\\\n' + '```json' + '\\\\n' + tool['function']['arguments'] + '\\\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}\", 'qwen2.attention.head_count_kv': '4', 'general.quantization_version': '2', 'tokenizer.ggml.model': 'gpt2', 'qwen2.feed_forward_length': '18944', 'qwen2.attention.head_count': '28', 'tokenizer.ggml.eos_token_id': '151643', 'qwen2.rope.freq_base': '10000.000000'}\n",
      "Available chat formats from metadata: chat_template.default\n",
      "Using gguf chat template: {% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}\n",
      "Using chat eos_token: <｜end▁of▁sentence｜>\n",
      "Using chat bos_token: <｜begin▁of▁sentence｜>\n"
     ]
    }
   ],
   "source": [
    "# Create the provider.\n",
    "llm = ChatLlamaCpp(\n",
    "    max_tokens=1024,\n",
    "    n_ctx=4096,\n",
    "    temperature=0.2,\n",
    "    n_gpu_layers=40,  # 使用 GPU 加速（如有）\n",
    "    model_path='/home/spike/code/AI/deepseek-r1-distill-qwen-7b-q4_k_m.gguf'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0657db9c-aa2e-4799-b88a-ec8b1d90714c",
   "metadata": {},
   "source": [
    "# RAG\n",
    "## 加载Embedding模型，打开向量数据库，找到最相似的查询结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4bdc9367-4ad1-4bc4-b832-fbdd534e41bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_model_load_from_file_impl: using device CUDA0 (NVIDIA GeForce RTX 3060) - 7396 MiB free\n",
      "llama_model_loader: loaded meta data with 23 key-value pairs and 101 tensors from ./all-MiniLM-L6-v2.F32.gguf (version GGUF V3 (latest))\n",
      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
      "llama_model_loader: - kv   0:                       general.architecture str              = bert\n",
      "llama_model_loader: - kv   1:                               general.name str              = all-MiniLM-L6-v2\n",
      "llama_model_loader: - kv   2:                           bert.block_count u32              = 6\n",
      "llama_model_loader: - kv   3:                        bert.context_length u32              = 512\n",
      "llama_model_loader: - kv   4:                      bert.embedding_length u32              = 384\n",
      "llama_model_loader: - kv   5:                   bert.feed_forward_length u32              = 1536\n",
      "llama_model_loader: - kv   6:                  bert.attention.head_count u32              = 12\n",
      "llama_model_loader: - kv   7:          bert.attention.layer_norm_epsilon f32              = 0.000000\n",
      "llama_model_loader: - kv   8:                          general.file_type u32              = 0\n",
      "llama_model_loader: - kv   9:                      bert.attention.causal bool             = false\n",
      "llama_model_loader: - kv  10:                          bert.pooling_type u32              = 1\n",
      "llama_model_loader: - kv  11:            tokenizer.ggml.token_type_count u32              = 2\n",
      "llama_model_loader: - kv  12:                tokenizer.ggml.bos_token_id u32              = 101\n",
      "llama_model_loader: - kv  13:                tokenizer.ggml.eos_token_id u32              = 102\n",
      "llama_model_loader: - kv  14:                       tokenizer.ggml.model str              = bert\n",
      "llama_model_loader: - kv  15:                      tokenizer.ggml.tokens arr[str,30522]   = [\"[PAD]\", \"[unused0]\", \"[unused1]\", \"...\n",
      "llama_model_loader: - kv  16:                      tokenizer.ggml.scores arr[f32,30522]   = [-1000.000000, -1000.000000, -1000.00...\n",
      "llama_model_loader: - kv  17:                  tokenizer.ggml.token_type arr[i32,30522]   = [3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...\n",
      "llama_model_loader: - kv  18:            tokenizer.ggml.unknown_token_id u32              = 100\n",
      "llama_model_loader: - kv  19:          tokenizer.ggml.seperator_token_id u32              = 102\n",
      "llama_model_loader: - kv  20:            tokenizer.ggml.padding_token_id u32              = 0\n",
      "llama_model_loader: - kv  21:                tokenizer.ggml.cls_token_id u32              = 101\n",
      "llama_model_loader: - kv  22:               tokenizer.ggml.mask_token_id u32              = 103\n",
      "llama_model_loader: - type  f32:  101 tensors\n",
      "print_info: file format = GGUF V3 (latest)\n",
      "print_info: file type   = all F32\n",
      "print_info: file size   = 86.08 MiB (32.00 BPW) \n",
      "init_tokenizer: initializing tokenizer for type 3\n",
      "load: control token:    101 '[CLS]' is not marked as EOG\n",
      "load: control token:    103 '[MASK]' is not marked as EOG\n",
      "load: control token:      0 '[PAD]' is not marked as EOG\n",
      "load: control token:    100 '[UNK]' is not marked as EOG\n",
      "load: control token:    102 '[SEP]' is not marked as EOG\n",
      "load: special_eos_id is not in special_eog_ids - the tokenizer config may be incorrect\n",
      "load: special tokens cache size = 5\n",
      "load: token to piece cache size = 0.2032 MB\n",
      "print_info: arch             = bert\n",
      "print_info: vocab_only       = 0\n",
      "print_info: n_ctx_train      = 512\n",
      "print_info: n_embd           = 384\n",
      "print_info: n_layer          = 6\n",
      "print_info: n_head           = 12\n",
      "print_info: n_head_kv        = 12\n",
      "print_info: n_rot            = 32\n",
      "print_info: n_swa            = 0\n",
      "print_info: n_embd_head_k    = 32\n",
      "print_info: n_embd_head_v    = 32\n",
      "print_info: n_gqa            = 1\n",
      "print_info: n_embd_k_gqa     = 384\n",
      "print_info: n_embd_v_gqa     = 384\n",
      "print_info: f_norm_eps       = 1.0e-12\n",
      "print_info: f_norm_rms_eps   = 0.0e+00\n",
      "print_info: f_clamp_kqv      = 0.0e+00\n",
      "print_info: f_max_alibi_bias = 0.0e+00\n",
      "print_info: f_logit_scale    = 0.0e+00\n",
      "print_info: f_attn_scale     = 0.0e+00\n",
      "print_info: n_ff             = 1536\n",
      "print_info: n_expert         = 0\n",
      "print_info: n_expert_used    = 0\n",
      "print_info: causal attn      = 0\n",
      "print_info: pooling type     = 1\n",
      "print_info: rope type        = 2\n",
      "print_info: rope scaling     = linear\n",
      "print_info: freq_base_train  = 10000.0\n",
      "print_info: freq_scale_train = 1\n",
      "print_info: n_ctx_orig_yarn  = 512\n",
      "print_info: rope_finetuned   = unknown\n",
      "print_info: ssm_d_conv       = 0\n",
      "print_info: ssm_d_inner      = 0\n",
      "print_info: ssm_d_state      = 0\n",
      "print_info: ssm_dt_rank      = 0\n",
      "print_info: ssm_dt_b_c_rms   = 0\n",
      "print_info: model type       = 22M\n",
      "print_info: model params     = 22.57 M\n",
      "print_info: general.name     = all-MiniLM-L6-v2\n",
      "print_info: vocab type       = WPM\n",
      "print_info: n_vocab          = 30522\n",
      "print_info: n_merges         = 0\n",
      "print_info: BOS token        = 101 '[CLS]'\n",
      "print_info: EOS token        = 102 '[SEP]'\n",
      "print_info: UNK token        = 100 '[UNK]'\n",
      "print_info: SEP token        = 102 '[SEP]'\n",
      "print_info: PAD token        = 0 '[PAD]'\n",
      "print_info: MASK token       = 103 '[MASK]'\n",
      "print_info: LF token         = 0 '[PAD]'\n",
      "print_info: EOG token        = 102 '[SEP]'\n",
      "print_info: max token length = 21\n",
      "load_tensors: loading model tensors, this can take a while... (mmap = true)\n",
      "load_tensors: layer   0 assigned to device CPU\n",
      "load_tensors: layer   1 assigned to device CPU\n",
      "load_tensors: layer   2 assigned to device CPU\n",
      "load_tensors: layer   3 assigned to device CPU\n",
      "load_tensors: layer   4 assigned to device CPU\n",
      "load_tensors: layer   5 assigned to device CPU\n",
      "load_tensors: layer   6 assigned to device CPU\n",
      "load_tensors: tensor 'token_embd.weight' (f32) (and 100 others) cannot be used with preferred buffer type CPU_AARCH64, using CPU instead\n",
      "load_tensors: offloading 0 repeating layers to GPU\n",
      "load_tensors: offloaded 0/7 layers to GPU\n",
      "load_tensors:   CPU_Mapped model buffer size =    86.08 MiB\n",
      "...............................\n",
      "llama_init_from_model: n_seq_max     = 1\n",
      "llama_init_from_model: n_ctx         = 512\n",
      "llama_init_from_model: n_ctx_per_seq = 512\n",
      "llama_init_from_model: n_batch       = 512\n",
      "llama_init_from_model: n_ubatch      = 512\n",
      "llama_init_from_model: flash_attn    = 0\n",
      "llama_init_from_model: freq_base     = 10000.0\n",
      "llama_init_from_model: freq_scale    = 1\n",
      "llama_kv_cache_init: kv_size = 512, offload = 1, type_k = 'f16', type_v = 'f16', n_layer = 6, can_shift = 1\n",
      "llama_kv_cache_init: layer 0: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 1: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 2: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 3: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 4: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init: layer 5: n_embd_k_gqa = 384, n_embd_v_gqa = 384\n",
      "llama_kv_cache_init:        CPU KV buffer size =     4.50 MiB\n",
      "llama_init_from_model: KV self size  =    4.50 MiB, K (f16):    2.25 MiB, V (f16):    2.25 MiB\n",
      "llama_init_from_model:        CPU  output buffer size =     0.00 MiB\n",
      "llama_init_from_model:      CUDA0 compute buffer size =    17.31 MiB\n",
      "llama_init_from_model:  CUDA_Host compute buffer size =     3.50 MiB\n",
      "llama_init_from_model: graph nodes  = 221\n",
      "llama_init_from_model: graph splits = 100 (with bs=512), 1 (with bs=1)\n",
      "CUDA : ARCHS = 860 | USE_GRAPHS = 1 | PEER_MAX_BATCH_SIZE = 128 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | AARCH64_REPACK = 1 | \n",
      "Model metadata: {'tokenizer.ggml.mask_token_id': '103', 'tokenizer.ggml.padding_token_id': '0', 'tokenizer.ggml.seperator_token_id': '102', 'tokenizer.ggml.unknown_token_id': '100', 'tokenizer.ggml.model': 'bert', 'tokenizer.ggml.eos_token_id': '102', 'general.architecture': 'bert', 'bert.block_count': '6', 'bert.attention.layer_norm_epsilon': '0.000000', 'bert.context_length': '512', 'bert.feed_forward_length': '1536', 'bert.embedding_length': '384', 'tokenizer.ggml.cls_token_id': '101', 'tokenizer.ggml.token_type_count': '2', 'bert.attention.head_count': '12', 'tokenizer.ggml.bos_token_id': '101', 'general.file_type': '0', 'general.name': 'all-MiniLM-L6-v2', 'bert.attention.causal': 'false', 'bert.pooling_type': '1'}\n",
      "Using fallback chat format: llama-2\n",
      "llama_perf_context_print:        load time =       4.06 ms\n",
      "llama_perf_context_print: prompt eval time =       2.97 ms /     7 tokens (    0.42 ms per token,  2357.70 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =       4.08 ms /     8 tokens\n"
     ]
    }
   ],
   "source": [
    "# create the open-source embedding function\n",
    "embedding_function = LlamaCppEmbeddings(model_path=\"./all-MiniLM-L6-v2.F32.gguf\")\n",
    "connection = SQLiteVec.create_connection(db_file=\"./lm.hlp\")\n",
    "db = SQLiteVec(\n",
    "    table=\"state_union\", embedding=embedding_function, connection=connection\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7da5517-bb1d-4268-bf60-d3c409f22169",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "llama_perf_context_print:        load time =       4.06 ms\n",
      "llama_perf_context_print: prompt eval time =       1.96 ms /     7 tokens (    0.28 ms per token,  3573.25 tokens per second)\n",
      "llama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\n",
      "llama_perf_context_print:       total time =       2.22 ms /     8 tokens\n"
     ]
    }
   ],
   "source": [
    "zh_db = SQLiteVec(\n",
    "    table=\"state_union\", embedding=embedding_function, connection=SQLiteVec.create_connection(db_file=\"./zh_kwds.db\")\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aa087f1b-7325-46a6-b947-8421e3ad73de",
   "metadata": {},
   "source": [
    "# 先用翻译模型加RAG对问题进行翻译，翻译成英文"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "998056b1-7f77-4326-987b-15ef84fc6dbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Llama.generate: 10 prefix-match hit, remaining 12 prompt tokens to eval\n",
      "llama_perf_context_print:        load time =     266.50 ms\n",
      "llama_perf_context_print: prompt eval time =      71.72 ms /    12 tokens (    5.98 ms per token,   167.32 tokens per second)\n",
      "llama_perf_context_print:        eval time =    7284.12 ms /   458 runs   (   15.90 ms per token,    62.88 tokens per second)\n",
      "llama_perf_context_print:       total time =    7734.32 ms /   470 tokens\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==================================\u001b[1m Ai Message \u001b[0m==================================\n",
      "\n",
      "<think>\n",
      "嗯，用户给了一个查询，让我 tokenize \"硬杆模型和软杆模型的区别\"，并用空格分隔中文短语。首先，我得弄清楚什么是 tokenization。一般来说，tokenization就是把连续的文字分割成有意义的单位，比如单词或短语。\n",
      "\n",
      "那这个句子有两个部分：\"硬杆模型\" 和 \"软杆模型\"，中间用“和”连接，最后是“区别”。用户要求用空格分隔中文短语，所以可能需要将整个句子分成几个部分。但这里有个问题，中文中的“和”有时候会被视为一个连接词，而不是分割点。\n",
      "\n",
      "我应该考虑是否把“硬杆模型和软杆模型的区别”分成四个部分：硬杆模型、和、软杆模型、区别。这样每个部分都是单独的 token。或者，用户可能希望将整个句子分成更小的部分，比如每个中文短语作为一个 token。\n",
      "\n",
      "另外，用户提到的是中文短语，所以可能需要保持“硬杆模型”和“软杆模型”作为整体，然后加上“的区别”。但中间的“和”是否应该分割呢？这取决于具体的 tokenizer 是否会处理这样的情况。通常， tokenizer 会将句子分成单词，而连接词如“和”可能会被保留在一起。\n",
      "\n",
      "所以，可能正确的 tokenization 是：硬杆模型 和 软杆模型 区别。或者，如果用户希望每个中文短语单独作为一个 token，那么可能是：硬杆模型、软杆模型、区别。但中间的“和”是否需要分割呢？这取决于 tokenizer 的处理方式。\n",
      "\n",
      "另外，考虑到用户可能是在进行自然语言处理任务，比如分词或特征提取，他们可能希望将整个句子分解成更小的单位，以便分析每个部分的意义。因此，把“硬杆模型”和“软杆模型”分开，加上“和”作为连接词，然后再加上“区别”，可能会更有帮助。\n",
      "\n",
      "总结一下，用户的需求可能是将整个句子分成四个 token：硬杆模型、和、软杆模型、区别。这样每个部分都是一个有意义的单位，方便后续处理。\n",
      "</think>\n",
      "\n",
      "\"硬杆模型 和 软杆模型 区别\"\n"
     ]
    }
   ],
   "source": [
    "message = \"硬杆模型和软杆模型的区别\"\n",
    "prompt =   [f\"tokenize \\“{message}\\\" ,split chinese phrase by space\"]\n",
    "response = llm.invoke(prompt, config={\"max_tokens\": 512})  # 动态覆盖配置)\n",
    "response.pretty_print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "bac354b4-32b2-4783-a1a3-c85f335bf498",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['如何使用三段式进行轨道设计']\n"
     ]
    }
   ],
   "source": [
    "print(tokenize)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e5331ed-6302-4435-8855-d5e7f5d836f4",
   "metadata": {},
   "source": [
    "# 调用onnx的演示，翻译结果不理想，可直接使用deepseek的中文翻译"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eeb43d7f-e06f-4558-a7d6-4db7bdca360f",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_content = \"\\n\\n\".join([ t.page_content for t in texts[:2]])\n",
    "system_message_content = (\n",
    "    \"You are a translator\"\n",
    "    \"Provide a short and direct answer without explanation\"\n",
    "    \"tranlate message to english\"\n",
    "    \"translate phrase in terms of the following dictionary strictly\"\n",
    "    \"{dictonary}\"\n",
    "    \"三段式=Slant\\n\"\n",
    "    \"优化对齐=OA\\n\"\n",
    "    \"五段式=Swell\\n\"\n",
    ")\n",
    "\n",
    "prompt = [SystemMessage(system_message_content)] + [message]\n",
    "# Example usage\n",
    "en_response = llm.invoke(prompt, config={\"max_tokens\": 512})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c26cbdbf-0929-4c69-b9d4-68631f38de8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "en_response.content"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40ad8f12-fa40-46fc-b4ac-cb5a96f45f59",
   "metadata": {},
   "source": [
    "from transformers import pipeline\n",
    "from optimum.onnxruntime import ORTModelForSeq2SeqLM\n",
    "from transformers import AutoTokenizer\n",
    "from langchain.llms import HuggingFacePipeline\n",
    "model_name = \"/AI/opus-mt-zh-en\"  # e.g., \"Helsinki-NLP/opus-mt-en-zh\"\n",
    "model = ORTModelForSeq2SeqLM.from_pretrained(model_name, subfolder='onnx',  encoder_file_name='encoder_model.onnx')\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "onnx_pipeline = pipeline(\n",
    "    \"text2text-generation\",\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    device_map=\"auto\",  # Use GPU if available\n",
    ")\n",
    "onnx_llm = HuggingFacePipeline(pipeline=onnx_pipeline)\n",
    "system_message_content = (\n",
    "    \"三段式=Slant\"\n",
    ")\n",
    "prompt = [system_message_content] + [\"介绍三段式轨道设计方法\"]\n",
    "en_response = onnx_llm.generate(prompt)\n",
    "print(en_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "937e869e-a392-41d4-99bb-bf03f79994e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#message = \"input parameters in optimum align\"\n",
    "docs = db.similarity_search(“Introduce the Slant-based three-segment design approach\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f32afc4d-d5b9-4cc4-9258-bfff09c70d6f",
   "metadata": {},
   "source": [
    "# 生成一段提示词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7022731-d8e5-4b90-bd7b-485cccc0d3d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Format into prompt\n",
    "docs_content = \"\\n\\n\".join([doc.page_content for doc in docs[:3]])\n",
    "system_message_content = (\n",
    "    \"You are an oil drilling assistant for question-answering tasks. \"\n",
    "    \"Use the following pieces of retrieved context to answer \"\n",
    "    \"Must use Chinese to provide a short and direct answer without explanation \"\n",
    "    \"\\n\\n\"\n",
    "    f\"{docs_content}\"\n",
    ")\n",
    "\n",
    "prompt = [SystemMessage(system_message_content)] + [\"Introduce the Slant-based three-segment design approach\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf73706-5acf-4005-9a67-7d433b086836",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48f555cc-8be5-4b6e-8c55-e47bd47cb15c",
   "metadata": {},
   "source": [
    "# 根据提示词，使用DeepSeek推理模型生成结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9591459-f2a6-4c25-97d6-bb7e6f307353",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run\n",
    "response = llm.invoke(prompt, config={\"max_tokens\": 512})  # 动态覆盖配置)\n",
    "response.pretty_print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
